import java.util.ArrayList;
import java.util.Queue;
import java.util.Set;
import java.util.StringTokenizer;

import jsoup.nodes.Document;
import jsoup.nodes.Element;
import jsoup.select.Elements;

/**
 * This is the class that parses the pages from the pageBuffer queue and puts
 * the URLS into a pageToRetrieve queue.
 * 
 * @author Silvia
 * 
 */
public class httpParser implements Runnable {
	/**
	 * The current webpage
	 */
	private WebPage page;
	/**
	 * URL queue, need to save reference for adding to it
	 */
	private Queue<String> urls;

	/**
	 * data gatherer, need to save reference for giving back stats.
	 */
	private DataGatherer dg;
	private ArrayList<String> keyWords;
	private int[] keyWordCount = new int[10];
	private Set<String> pagesVisited;
	private Set<String> disallowed;

	public httpParser(Queue<WebPage> bufferQueue, Queue<String> urlQueue,
			DataGatherer gatherer, ArrayList<String> words, Set<String> visited, 
			Set<String> disallowList) {
		if (!bufferQueue.isEmpty()) {
			page = bufferQueue.remove();
		}
		disallowed = disallowList;
		urls = urlQueue;
		dg = gatherer;
		keyWords = words;
		pagesVisited = visited;
	}

	@Override
	public void run() {
		long timeBefore = System.currentTimeMillis();
		// get all the links of different types: frame src or a href as
		// htm/html/txt links
		if (page != null) {

			Document doc = page.getPageData();
			Elements htm = doc.select("a[href$=.htm]");
			Elements fhtm = doc.select("frame[src$=.htm]");
			Elements html = doc.select("a[href$=.html]");
			Elements fhtml = doc.select("frame[src$=.html]");
			Elements txt = doc.select("a[href$=.txt]");
			Elements ftxt = doc.select("frame[src$=.txt]");
			Elements main = doc.select("a[href$=/]");
			Elements fmain = doc.select("frame[href$=/]");

			int count = 0;

			for (Element page : htm) {
				String absUrl = page.absUrl("href");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					count++;
				}
			}
			for (Element page : fhtm) {
				String absUrl = page.absUrl("src");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			for (Element page : html) {
				String absUrl = page.attr("abs:href");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			for (Element page : fhtml) {
				String absUrl = page.absUrl("src");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			for (Element page : txt) {
				String absUrl = page.absUrl("href");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			for (Element page : ftxt) {
				String absUrl = page.absUrl("src");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			for (Element page : main) {
				String absUrl = page.absUrl("src");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			for (Element page : fmain) {
				String absUrl = page.absUrl("src");
				if (absUrl.length() > 5 && !pagesVisited.contains(absUrl)
						&& !disallowed.contains(absUrl)) {
					urls.offer(absUrl);
					pagesVisited.add(absUrl);
					count++;
				}
			}
			
			try {
			String bodyText = doc.body().text();
			//Scanner pageScanner = new Scanner(bodyText);
			StringTokenizer pageScanner = new StringTokenizer(bodyText);
			int wordCount = 0;
			
			//while (pageScanner.hasNext()) {
			while (pageScanner.hasMoreTokens()) {
				wordCount++;
				//String current = pageScanner.next();
				String current = pageScanner.nextToken();
				if (current.length() < 20) {
					current = current.replaceAll("\\W", "");
				}

				for (int i = 0; i < keyWords.size(); i++) {
					if (keyWords.get(i).equalsIgnoreCase(current)) {
						keyWordCount[i]++;
					}
				}
			}

			long timeAfter = System.currentTimeMillis();
			long duration = timeAfter - timeBefore;
			
			//pageScanner.close();
			dg.update(page.getURL(), keyWordCount, count, wordCount, duration);
			} catch (NullPointerException e) {
				// die
			}

		}

	}
}
